/*******************************************************************************
 * Copyright (c) 2018, 2018 GuoGuang Corporation and others.
 * All rights reserved. 
 *  
 * Create By Time 2018-2018年3月20日-下午1:31:13
 * Contributors:
 *     longshiyu
 * @longsy316@163.com
 *******************************************************************************/

package com.lsy.onehot.rnn.qa;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.gg.lsy.ai.utils.words.StringUtils;
import com.lsy.lstm.ai.rnn.qa.dataset.io.CorpusHandle;
import com.lsy.lstm.ai.rnn.qa.interfs.ILSTMRnn_AIQAConst;
import com.lsy.onehot.rnn.qa.data.DataSetHandle;
import com.lsy.onehot.rnn.qa.inner.ICurpusMateSet;
import com.lsy.onehot.rnn.qa.inner.IOnehot;

/**
 * @author Administrator
 * @time 2018 - 2018年3月20日 - 下午1:31:13
 * @desc CurpusProcess.java
 */
public class CurpusProcess implements IOnehot,ICurpusMateSet {
	/**
	 * 
	 */
	public CurpusProcess() {
		// TODO Auto-generated constructor stub
	}
	public  Set<Character> dictSpecialSet = new HashSet<>() ;
	public  LinkedList<Character> dictSpeciallist = new LinkedList<Character>();
	public  Map<Long, String > dict_Q = new HashMap<>();
	public  Map<Long, String> dict_A = new HashMap<>();
    
    public LinkedList<Character> getListChar(){
    	return dictSpeciallist ;
    }
    
    public Map<Long, String> getDict_A(){
    	return dict_A;
    }

    public Map<Long, String> getDict_Q(){
    	return dict_Q;
    }
    
	public int getInputNums(){
		return dictSpeciallist.size() ;
	}
	
	public void handleSigalDict() throws IOException{
        try (BufferedReader br = CorpusHandle.readFile( IOnehot.DICT_SPELICAL_FILE )) {
        	String line;
        	while( ( line = br.readLine() ) != null  ){
        		char[] ch = line.toCharArray();
        		for(char c :ch)
        		{
        			if(StringUtils.isContainChinese(c) )
        			{
        				dictSpecialSet.add(c);
        			}
        		}
        		
    		}
    	}
        handleSigalDict2();
	}
	
	public void handleSigalDict2() throws IOException{
        try (BufferedReader br = CorpusHandle.readFile( IOnehot.DICT_MARK )) {
        	String line;
        	while( ( line = br.readLine() ) != null  ){
        		char[] ch = line.toCharArray();
        		for(char c :ch)
        		{
        			dictSpeciallist.add(c);
        		}
    		}
    	}
        
    	dictSpeciallist.addAll(dictSpecialSet) ;
    	DataSetHandle.saveAsSignalVectorModel(dictSpeciallist);
	}
	
	//50w.conv 解析
	public void handleDict() throws IOException{
        try (BufferedReader br = CorpusHandle.readFile( IOnehot.DICT_SPELICAL_FILE )) {
        	String line;
        	long i = 0;
        	long j = 0;
        	while( ( line = br.readLine() ) != null  ){
        		if( !line.startsWith("E") ){
        			process(line , dict_Q, i );
        			i++;
        			if( ( line = br.readLine() ) != null )
        				process(line , dict_A, j );
	        			j++;
            		}
        		}
    	}
	}
	
	//DICT_SPELICAL_FILE_goossip
	public void handleDict_Goossip() throws IOException{
        try (BufferedReader br = CorpusHandle.readFile( IOnehot.DICT_SPELICAL_FILE )) {
        	String line;
        	long i = 0;
        	long j = 0;
        	while( ( line = br.readLine() ) != null  ){
        		if( !line.startsWith("E") ){
        			process(line , dict_Q, i );
        			i++;
        			if( ( line = br.readLine() ) != null )
        				process(line , dict_A, j );
	        			j++;
            		}
        		}
    	}
	}
	
	private void process(String line, Map<Long, String> courpus,long count ){
		line = line.replaceFirst("m", "").replaceAll("\\s*","");
		courpus.put(count ,line);
	}
	 
	
	/**
	* @author longsy
	* @time : 20182018年3月20日下午1:31:13
	* @desc CurpusProcess.java
	* @param args
	 * @throws Exception 
	*/
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
		
		CurpusProcess cur = new CurpusProcess();
		cur.handleSigalDict(); 
		cur.handleDict() ;
//		
//		cur.dictSpecialSet.add("a");
//		cur.dictSpecialSet.add("a");
//		cur.dictSpecialSet.add("c");
//		cur.dictSpecialSet.add("c");
//		cur.dictSpecialSet.add("a");
//		
//		System.out.println("i:" +  cur.dict_Q.size() );
//	 
//		
//		OneHotProcess onehot = new OneHotProcess();
//		onehot.setICurpusMateSet(cur);
		
//		File out = new File("src/main/resources/one-hot-q.txt") ;
//		
//		onehot.outputAnswerVecQ();
//		onehot.outputAnswerVecA();
		
	}

}
